from bs4 import BeautifulSoup
import requests
import json
import math
import crawler
import urllib.parse
from pymongo import MongoClient

client=MongoClient('localhost',27017)
db=client.cache
headers={}
headers['User-agent'] = 'wzwp'

def getuserdetail(username):
    proxy=crawler.getproxy()
    username=urllib.parse.quote(username)
    url='https://www.baidu.com/p/%s/detail'%username
    #html=requests.get().text
    html=crawler.download(url,headers=headers,proxy=proxy,num_retries=-1)
    
    bs=BeautifulSoup(html,'html.parser')
    userprofiles=bs.find_all(attrs={'class':'profile-attr'})
    dic={}
    for up in userprofiles:
        attr=str(up.next_element.next_element.next_element)
        attr=attr.replace('<span class="profile-cnt">','').replace('</span>','').replace(' ','').replace('<span>','')
        dic[up.text]=attr
    print(dic)
    return dic

def userpop():
    record=db.tiabuser1.find_and_modify(
            query={'status1':0},
            update={'$set':{'status1':1}}
        )
        # record=self.db.tiabalist.find_one()
    if record:
        return record['_id']
    else:
        return False



username=userpop()
while username:
    try:     
        #print(username)
        proxy=crawler.getproxy()
        username1=urllib.parse.quote(username)
        url='https://www.baidu.com/p/%s/detail'%username1
        #html=requests.get().text
        html=crawler.download(url,headers=headers,proxy=proxy,num_retries=-1)
        dic={}
        if html:
            bs=BeautifulSoup(html,'html.parser')
            userprofiles=bs.find_all(attrs={'class':'profile-attr'})
            
            for up in userprofiles:
                attr=str(up.next_element.next_element.next_element)
                attr=attr.replace('<span class="profile-cnt">','').replace('</span>','').replace(' ','').replace('<span>','')
                if up.text:
                    dic[up.text]=attr            
            if dic:
                dic['status1']=2
            else:
                dic['status1']=4
        else:
            dic['status1']=3
        print('%s,%s'%(username,dic['status1']))
        db.tiabuser1.update({'_id':username},{'$set':dic})
        username=userpop()
    except Exception as identifier:
        print('%s,%s'%(username,'error'))
        db.tiabuser1.update({'_id':username},{'$set':{'status1':3}})
        username=userpop()